import re
import requests
import json
from multiprocessing import Pool

def get_one_page(url):
    header = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0'}
    response = requests.get(url, headers=header)
    if response.status_code == 200:
        return response.text
    return None

def parse_one_page(html):
    pattern = re.compile('<dd>.*?board-index.*?">(\d+)</i>.*?name.*?<a.*?href="(.*?)".*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>.*?score.*?integer">(.*?)</i>.*?fraction">(\d+)</i>.*?</dd>', re.S)
    results = re.findall(pattern, html)
    for result in results:
        url = 'http://maoyan.com' + result[1]
        response = get_one_page(url)
        #print(response)
        pattern2 = re.compile('<div.*?mod-content.*?dra">(.*?)</span>', re.S)
        res = re.findall(pattern2, response)
        yield {
            'index': result[0],
            'url': 'http://maoyan.com' + result[1],
            'title': result[2],
            'actor': result[3].strip()[3:],
            'time': result[4].strip()[5:],
            'score': result[5] + result[6],
            'more': res[0]
        }

def write_in_file(content):
    with open('maoyan.json', 'a', encoding='utf8') as f:
        f.write(json.dumps(content, ensure_ascii=False)+',\n')
        f.close()

def main(offset):
    url = 'http://maoyan.com/board/4?offset=' + str(offset)
    html = get_one_page(url)
    result = parse_one_page(html)
    for res in result:
        print(res)
        write_in_file(res)

if __name__ == "__main__":
    pool = Pool()
    pool.map(main, [i*10 for i in range(10)])
